{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import os\n",
    "import cv2\n",
    "import numpy as np\n",
    "import os\n",
    "from matplotlib import pyplot as plt\n",
    "import random\n",
    "import pandas as pd\n",
    "import pickle\n",
    "# import xml.etree.ElementTree as ET\n",
    "import scipy.io as scio\n",
    "from utils.boxes import bbox_overlaps\n",
    "from PIL import Image\n",
    "import json\n",
    "from pprint import pprint\n",
    "import sng_parser\n",
    "import ipdb\n",
    "\n",
    "import torch\n",
    "# from pytorch_pretrained_bert.tokenization import BertTokenizer\n",
    "# from pytorch_pretrained_bert.modeling import BertModel\n",
    "from skipthoughts import UniSkip"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# # create Selective Search Segmentation Object using default parameters\n",
    "# cv2.setUseOptimized(True);\n",
    "# cv2.setNumThreads(4);\n",
    "# ss = cv2.ximgproc.segmentation.createSelectiveSearchSegmentation()\n",
    "# def ss_boxes(img, mode='f', keep_boxes=100):\n",
    "#     # read image\n",
    "#     # set input image on which we will run segmentation\n",
    "#     ss.setBaseImage(img)\n",
    " \n",
    "#     # Switch to fast but low recall Selective Search method\n",
    "#     if (mode == 'f'):\n",
    "#         ss.switchToSelectiveSearchFast()\n",
    "#     # Switch to high recall but slow Selective Search method\n",
    "#     elif (mode == 'q'):\n",
    "#         ss.switchToSelectiveSearchQuality()\n",
    " \n",
    "#     # run selective search segmentation on input image\n",
    "#     rects = ss.process()[:keep_boxes, :]\n",
    "# #     print('Total Number of Region Proposals: {}'.format(len(rects)))\n",
    "#     rects[:, 2] += rects[:, 0]\n",
    "#     rects[:, 3] += rects[:, 1]\n",
    "#     return rects"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "sentence_root = '/p300/flickr30k_images/flickr30k_anno/sent_anno.json'\n",
    "bbox_root = '/p300/flickr30k_images/flickr30k_anno/box_anno.json'\n",
    "im_root = '/p300/flickr30k_images/flickr30k_images/'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "sent_anno = json.load(open(sentence_root, 'r'))\n",
    "bbox_anno = json.load(open(bbox_root, 'r'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "im_ids = list(sent_anno.keys())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Denoise"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# errors = np.array([129602216, 5,\n",
    "# 133010954, 5,\n",
    "# 153299765, 2,\n",
    "# 158388874, 5,\n",
    "# 200767661, 5,\n",
    "# 3367384342, 2,\n",
    "# 4671832284, 1,\n",
    "# 7638876050, 5,\n",
    "# 3652094744, 1,\n",
    "# 7017792809, 5,\n",
    "# 7052778829, 5,\n",
    "# 7232861768, 5,\n",
    "# 4901396689, 5,\n",
    "# 6154676236, 5,\n",
    "# 6442477951, 5,\n",
    "# 4351142771, 5,\n",
    "# 5096654568, 5,\n",
    "# 100759042, 5,\n",
    "# 5566135246, 5,\n",
    "# 2190899457, 5,\n",
    "# 180753784, 3])\n",
    "# errors = errors.reshape(len(errors)//2,2)\n",
    "# for (im_id, sent_id) in errors:\n",
    "#     del sent_anno[str(im_id)][sent_id-1]\n",
    "# with open(sentence_root, 'w') as f:\n",
    "#     json.dump(sent_anno, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# sent_anno_new = {}\n",
    "\n",
    "# for im_id, all_sents in sent_anno.items():\n",
    "#     boxes = bbox_anno[im_id]\n",
    "#     all_phrase_ids = []\n",
    "#     for k, v in boxes['boxes'].items():\n",
    "#         all_phrase_ids.append(k)\n",
    "    \n",
    "#     all_sents_new = []\n",
    "#     for sent_id, sent in enumerate(all_sents):\n",
    "#         phrases = sent['phrases']\n",
    "#         phrase_ids = []\n",
    "#         for phr in phrases:\n",
    "#             if phr['phrase_id'] not in all_phrase_ids:\n",
    "#                 continue\n",
    "#             phrase_ids.append(phr['phrase_id'])\n",
    "        \n",
    "#         if len(phrase_ids) > 0:\n",
    "#             all_sents_new.append(sent)\n",
    "#     sent_anno_new[im_id] = all_sents_new\n",
    "            \n",
    "# with open(sentence_root, 'w') as f:\n",
    "#     json.dump(sent_anno_new, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_f = '/p300/flickr30k_images/split/train.txt'\n",
    "# test_f = '/p300/flickr30k_images/split/test.txt'\n",
    "# val_f = '/p300/flickr30k_images/split/val.txt'\n",
    "\n",
    "# cur_file = val_f\n",
    "# with open(cur_file) as ff:\n",
    "#     cur = ff.readlines()\n",
    "    \n",
    "# with open(cur_file+'.tmp', 'w') as fff:\n",
    "#     for i in cur:\n",
    "#         if i.strip() in im_ids:\n",
    "#             fff.write(i)\n",
    "            \n",
    "# with open(cur_file+'.tmp', 'w') as fff:\n",
    "#     for i, value in enumerate(cur):\n",
    "#         im_id = value.strip()\n",
    "#         sent_anno_num = len(sent_anno[im_id])\n",
    "#         for j in range(sent_anno_num):\n",
    "#             fff.write(im_id+'\\t'+str(j)+'\\n')\n",
    "            \n",
    "            \n",
    "# sent_anno_new = {}\n",
    "# for im_id, sent in sent_anno.items():\n",
    "#     if len(sent) > 0:\n",
    "#         sent_anno_new[im_id] = sent\n",
    "# with open(sentence_root, 'w') as f:\n",
    "#     json.dump(sent_anno_new, f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1000"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "with open('/p300/flickr30k_images/split/val_clean.txt') as ff:\n",
    "    res = ff.readlines()\n",
    "len(res)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### upper bound"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [],
   "source": [
    "vg_pred_root = '/p300/flickr30k_images/flickr30k_anno/precompute_proposals_nms_1e4.json'\n",
    "vg_pred = json.load(open(vg_pred_root))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "31783"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(vg_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(100, 5)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "np.array(vg_pred['100652400']).shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "all_gt_num = 0\n",
    "all_recall_num = 0\n",
    "for im_id in im_ids:\n",
    "    # get sentence and sentence anno\n",
    "    st_anno = sent_anno[im_id]\n",
    "    # get annotation\n",
    "    gt_anno = bbox_anno[im_id]\n",
    "    \n",
    "    # get gt boxes\n",
    "    gt_boxes = []\n",
    "    gt_box_ids = []\n",
    "    for pid, v in gt_anno['boxes'].items():\n",
    "        gt_box_ids.append(pid)\n",
    "        if len(v) == 1:\n",
    "            gt_boxes.append(v[0])\n",
    "        else:\n",
    "            # when a phrase respond to multiple regions, we take the union of them as paper given\n",
    "            v = np.array(v)\n",
    "            box = [v[:,0].min(), v[:,1].min(), v[:,2].max(), v[:,3].max()]\n",
    "            gt_boxes.append(box)\n",
    "    gt_boxes = np.array(gt_boxes)\n",
    "    \n",
    "    gt_num = gt_boxes.shape[0]\n",
    "    all_gt_num += gt_num\n",
    "    \n",
    "    ## given ss boxes from https://github.com/kanchen-usc/QRC-Net/blob/master/feature/README.md\n",
    "    #     rects = scio.loadmat(ss_boxes_root + im_id + '.mat')['cur_bbxes']\n",
    "    \n",
    "    ## ss boxes we compute\n",
    "#     im = cv2.imread(im_root + im_id + '.jpg')\n",
    "#     rects = ss_boxes(im, mode='f', keep_boxes=100)\n",
    "\n",
    "    ## boxes from pretrained resnet101-fpn on vg dataset\n",
    "#     rects = vg_pred[im_id].bbox.numpy()\n",
    "#     scores = vg_pred[im_id].extra_fields['scores']\n",
    "    rects = np.array(vg_pred[im_id])[:,:4]\n",
    "    \n",
    "    overlaps = bbox_overlaps(gt_boxes.astype(np.float32), rects.astype(np.float32))\n",
    "    recall_num = np.sum(overlaps.max(1)>0.5)\n",
    "    all_recall_num += recall_num"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193020 174496 0.90403067039685\n"
     ]
    }
   ],
   "source": [
    "print(all_gt_num, all_recall_num, all_recall_num/all_gt_num)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### NLP"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "parser = sng_parser.Parser('spacy', model='en')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "A woman in a red long-sleeved shirt and blue jeans stands at the lectern speaking at or about GLAM U WIKI .\n",
      "[{'first_word_index': 0,\n",
      "  'phrase': 'A woman',\n",
      "  'phrase_id': '142915',\n",
      "  'phrase_type': ['people']},\n",
      " {'first_word_index': 3,\n",
      "  'phrase': 'a red long-sleeved shirt',\n",
      "  'phrase_id': '142918',\n",
      "  'phrase_type': ['clothing']},\n",
      " {'first_word_index': 8,\n",
      "  'phrase': 'blue jeans',\n",
      "  'phrase_id': '142920',\n",
      "  'phrase_type': ['clothing']},\n",
      " {'first_word_index': 12,\n",
      "  'phrase': 'the lectern',\n",
      "  'phrase_id': '142919',\n",
      "  'phrase_type': ['other']},\n",
      " {'first_word_index': 18,\n",
      "  'phrase': 'GLAM U WIKI',\n",
      "  'phrase_id': '142916',\n",
      "  'phrase_type': ['other']}]\n",
      "Entities:\n",
      "+-------------+-----------------------------------------+---------------+\n",
      "| Head        | Span                                    | Modifiers     |\n",
      "|-------------+-----------------------------------------+---------------|\n",
      "| woman       | a woman                                 | a             |\n",
      "| jeans       | a red long-sleeved shirt and blue jeans | a,red,sleeved |\n",
      "| lectern     | the lectern                             | the           |\n",
      "| u glam wiki | glam u wiki                             |               |\n",
      "+-------------+-----------------------------------------+---------------+\n",
      "Relations:\n",
      "+-----------+------------+----------+\n",
      "| Subject   | Relation   | Object   |\n",
      "|-----------+------------+----------|\n",
      "| woman     | in         | jeans    |\n",
      "| woman     | at         | lectern  |\n",
      "+-----------+------------+----------+\n"
     ]
    }
   ],
   "source": [
    "im_id = im_ids[1000]\n",
    "st_anno = sent_anno[im_id]\n",
    "sentence = st_anno[0]['sentence']\n",
    "print(sentence)\n",
    "phrases = st_anno[0]['phrases']\n",
    "pprint(phrases)\n",
    "graph = parser.parse(sentence)\n",
    "sng_parser.tprint(graph)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "193020 158352 0.8203916692570719\n",
      "{'people': 51986, 'other': 36446, 'bodyparts': 6874, 'scene': 19335, 'clothing': 34635, 'vehicles': 3750, 'instruments': 1062, 'animals': 4251, None: 13}\n",
      "{'bodyparts': 5036, 'clothing': 6615, 'scene': 3023, 'animals': 444, 'people': 3995, 'other': 14283, 'vehicles': 585, 'instruments': 683, None: 4}\n"
     ]
    }
   ],
   "source": [
    "all_gt_num = 0\n",
    "all_recall_num = 0\n",
    "error_types = {}\n",
    "correct_types = {}\n",
    "for im_id in im_ids:\n",
    "    st_anno = sent_anno[im_id]\n",
    "    phrase_type_dict = {}\n",
    "    for sent in st_anno:\n",
    "        phrases = sent['phrases']\n",
    "        for phrase in phrases:\n",
    "            phrase_type_dict[phrase['phrase_id']] = phrase['phrase_type'][0]\n",
    "\n",
    "    gt_anno = bbox_anno[im_id]\n",
    "\n",
    "    # get gt boxes\n",
    "    gt_boxes = []\n",
    "    gt_box_type = []\n",
    "    for pid, v in gt_anno['boxes'].items():\n",
    "        gt_box_type.append(phrase_type_dict.get(pid))\n",
    "        if len(v) == 1:\n",
    "            gt_boxes.append(v[0])\n",
    "        else:\n",
    "            # when a phrase respond to multiple regions, we take the union of them as paper given\n",
    "            v = np.array(v)\n",
    "            box = [v[:,0].min(), v[:,1].min(), v[:,2].max(), v[:,3].max()]\n",
    "            gt_boxes.append(box)\n",
    "    gt_boxes = np.array(gt_boxes)\n",
    "\n",
    "    gt_num = gt_boxes.shape[0]\n",
    "    all_gt_num += gt_num\n",
    "\n",
    "    rects = vg_pred[im_id+'.jpg'].bbox.numpy()\n",
    "\n",
    "    overlaps = bbox_overlaps(gt_boxes.astype(np.float32), rects[:100, :].astype(np.float32))\n",
    "    pos_ind = np.where(overlaps.max(1)>=0.5)[0]\n",
    "    \n",
    "    recall_num = len(pos_ind)\n",
    "    all_recall_num += recall_num\n",
    "    \n",
    "    \n",
    "    for psid in pos_ind:\n",
    "        tmp_num = correct_types.setdefault(gt_box_type[psid], 0)\n",
    "        tmp_num += 1\n",
    "        correct_types[gt_box_type[psid]] = tmp_num\n",
    "    \n",
    "    \n",
    "    neg_ind = np.where(overlaps.max(1)<0.5)[0]\n",
    "    for ngid in neg_ind:\n",
    "        tmp_num = error_types.setdefault(gt_box_type[ngid], 0)\n",
    "        tmp_num += 1\n",
    "        error_types[gt_box_type[ngid]] = tmp_num\n",
    "print(all_gt_num, all_recall_num, all_recall_num/all_gt_num)\n",
    "print(correct_types)\n",
    "print(error_types)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "tokenizer = BertTokenizer.from_pretrained(\"bert-base-uncased\")\n",
    "model = BertModel.from_pretrained(\"bert-base-uncased\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "dir_st = 'data/skip-thoughts'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20214\n"
     ]
    }
   ],
   "source": [
    "# im_id = im_ids[0]\n",
    "vocab = set()\n",
    "for im_id in im_ids:\n",
    "    st_anno = sent_anno[im_id]\n",
    "    for sent in st_anno:\n",
    "        seq = sent['sentence'].lower()\n",
    "        tokenized_seq = seq.split(' ')\n",
    "\n",
    "        vocab.update(tokenized_seq)\n",
    "vocab = list(vocab)\n",
    "print(len(vocab))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "metadata": {},
   "outputs": [],
   "source": [
    "vocab_to_id = {v:i+1 for i,v in enumerate(vocab)}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 74,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warning: 3315/930911 words are not in dictionary, thus set UNK\n"
     ]
    }
   ],
   "source": [
    "uniskip = UniSkip(dir_st, vocab)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "metadata": {},
   "outputs": [],
   "source": [
    "error_count = 0\n",
    "\n",
    "im_id = im_ids[0]\n",
    "st_anno = sent_anno[im_id]\n",
    "phrase_type_dict = {}\n",
    "for sent in st_anno:\n",
    "    seq = sent['sentence'].lower()\n",
    "    tokenized_seq = seq.split(' ')\n",
    "\n",
    "    phrases = sent['phrases']\n",
    "    \n",
    "    inputs = torch.zeros((len(phrases), len(tokenized_seq))).long()\n",
    "    lengths = []\n",
    "    \n",
    "    for pid, phr in enumerate(phrases):\n",
    "        tokenized_phr = phr['phrase'].lower().split(' ')\n",
    "        phr_len = len(tokenized_phr)\n",
    "        lengths.append(phr_len)\n",
    "        \n",
    "        start_ind = phr['first_word_index']\n",
    "        end_ind = start_ind + phr_len\n",
    "        \n",
    "        phr_idx = []\n",
    "        for w in tokenized_phr:\n",
    "            phr_idx.append(vocab_to_id[w])\n",
    "        \n",
    "        inputs[pid][:phr_len] = torch.Tensor(phr_idx).long()\n",
    "        \n",
    "        assert tokenized_seq[start_ind:end_ind] == tokenized_phr\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "tensor([[ 9278, 18539,     0,     0,     0,     0,     0,     0,     0,     0],\n",
      "        [ 9278, 15584,     0,     0,     0,     0,     0,     0,     0,     0],\n",
      "        [15311,  3098,     0,     0,     0,     0,     0,     0,     0,     0]]) [2, 2, 2]\n"
     ]
    }
   ],
   "source": [
    "print(inputs, lengths)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "torch.Size([3, 2400])\n"
     ]
    }
   ],
   "source": [
    "embeds = uniskip(inputs, lengths=lengths)\n",
    "print(embeds.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# error_count = 0\n",
    "# for im_id in im_ids:\n",
    "# #     im_id = im_ids[0]\n",
    "#     st_anno = sent_anno[im_id]\n",
    "#     phrase_type_dict = {}\n",
    "#     for sent in st_anno:\n",
    "#         seq = sent['sentence'].lower()\n",
    "#         seq = \"[CLS] \" + seq + \" [SEP]\"\n",
    "#         tokenized_seq = seq.split(' ')\n",
    "#         try:\n",
    "#             indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_seq)\n",
    "#         except:\n",
    "#             ipdb.set_trace()\n",
    "# #         tokens_tensor = torch.tensor(indexed_tokens)\n",
    "\n",
    "#         phrases = sent['phrases']\n",
    "#         for phr in phrases:\n",
    "#             tokenized_phr = phr['phrase'].lower().split(' ')\n",
    "#             start_ind = phr['first_word_index'] + 1\n",
    "#             end_ind = start_ind + len(tokenized_phr)\n",
    "#             if tokenized_seq[start_ind:end_ind] != tokenized_phr:\n",
    "#     #             print(tokenized_seq[start_ind:end_ind])\n",
    "#     #             print(tokenized_phr)\n",
    "#                 error_count += 1\n",
    "            \n",
    "                \n",
    "# print(error_count)\n",
    "# #         assert tokenized_seq[start_ind:end_ind] == tokenized_phr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['a', 'harness']"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokenized_phr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['[CLS]',\n",
       " 'A',\n",
       " 'woman',\n",
       " 'dressed',\n",
       " 'in',\n",
       " 'black',\n",
       " 'is',\n",
       " 'walking',\n",
       " 'her',\n",
       " 'son',\n",
       " 'with',\n",
       " 'a',\n",
       " 'harness',\n",
       " '.',\n",
       " '[SEP]']"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "seq.split(' ')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'sentence': 'A man walked across a street holding a basket on top of his head filled with fruit and vegetables with one hand , while holding bags of grapes in his right hand .',\n",
       " 'phrases': [{'first_word_index': 0,\n",
       "   'phrase': 'A man',\n",
       "   'phrase_id': '84695',\n",
       "   'phrase_type': ['people']},\n",
       "  {'first_word_index': 4,\n",
       "   'phrase': 'a street',\n",
       "   'phrase_id': '84710',\n",
       "   'phrase_type': ['scene']},\n",
       "  {'first_word_index': 7,\n",
       "   'phrase': 'a basket',\n",
       "   'phrase_id': '84696',\n",
       "   'phrase_type': ['other']},\n",
       "  {'first_word_index': 12,\n",
       "   'phrase': 'his head',\n",
       "   'phrase_id': '84701',\n",
       "   'phrase_type': ['bodyparts']},\n",
       "  {'first_word_index': 16,\n",
       "   'phrase': 'fruit',\n",
       "   'phrase_id': '84706',\n",
       "   'phrase_type': ['other']},\n",
       "  {'first_word_index': 18,\n",
       "   'phrase': 'vegetables',\n",
       "   'phrase_id': '84696',\n",
       "   'phrase_type': ['other']},\n",
       "  {'first_word_index': 20,\n",
       "   'phrase': 'one hand',\n",
       "   'phrase_id': '84708',\n",
       "   'phrase_type': ['bodyparts']},\n",
       "  {'first_word_index': 25,\n",
       "   'phrase': 'bags of grapes',\n",
       "   'phrase_id': '84699',\n",
       "   'phrase_type': ['other']},\n",
       "  {'first_word_index': 29,\n",
       "   'phrase': 'his right hand',\n",
       "   'phrase_id': '84702',\n",
       "   'phrase_type': ['bodyparts']}]}"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "st_anno[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "pose",
   "language": "python",
   "name": "pose"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
